Import Necessary Libraries¶
In [1]:
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
from matplotlib.image import imread
from sklearn.linear_model import LinearRegression
from sklearn.model_selection import train_test_split
from sklearn.cluster import KMeans
from scipy.stats import zscore
from scipy.spatial.distance import cdist
from sklearn.cluster import AgglomerativeClustering
from scipy.spatial.distance import pdist
from scipy.cluster.hierarchy import cophenet, dendrogram, linkage
from sklearn import svm
from sklearn.decomposition import PCA, IncrementalPCA
import datetime
from sklearn.experimental import enable_iterative_imputer
from sklearn.impute import IterativeImputer
from sklearn.impute import SimpleImputer
%matplotlib inline
sns.set(color_codes=True)
Part I¶
Import and Warehouse data¶
Importing the data¶
In [2]:
df1=pd.read_csv('Part1 - Car name.csv')
In [3]:
print('The size of the data:', df1.size)
print('No of rows in the data:', df1.shape[0])
print('No of columns in the data:', df1.shape[1])
In [4]:
df2=pd.read_json('Part1 - Car-Attributes.json')
In [5]:
print('The size of the data:', df2.size)
print('No of rows in the data:', df2.shape[0])
print('No of columns in the data:', df2.shape[1])
Merging the Data¶
In [6]:
df3=df1.join(df2,how='left')
In [7]:
df3.head(10)
Out[7]:
In [8]:
print('The size of the data:', df3.size)
print('No of rows in the data:', df3.shape[0])
print('No of columns in the data:', df3.shape[1])
Saving the data¶
In [9]:
df3.to_csv('Merge_data.csv')
Importing the saved data¶
In [10]:
df=pd.read_csv('Merge_data.csv', index_col=0)
In [11]:
df.head(10)
Out[11]:
In [12]:
print('The size of the data:', df.size)
print('No of rows in the data:', df.shape[0])
print('No of columns in the data:', df.shape[1])
Data Cleansing¶
Dropping car_names attribute¶
In [13]:
df2=df.copy(deep=True)
In [14]:
df2.drop(axis=1, columns='car_name', inplace=True)
In [15]:
df2.columns
Out[15]:
Checking for null values¶
Converting all values to numeric in case if there are any string values¶
In [16]:
df2 = df2.convert_objects(convert_numeric=True)
In [17]:
df2.isnull().sum()
Out[17]:
It seems that thre are six null values in 'hp' column¶
In [18]:
medianFiller = lambda x: x.fillna(x.median())
df2 = df2.apply(medianFiller,axis=0)
Replacing the null values with median data¶
In [19]:
df2.isnull().sum()
Out[19]:
No null values now¶
Checking for zero values¶
In [20]:
(df2==0).all()
Out[20]:
None of the columns have zero values or entirely filled with zero¶
Checking correlation between attributes¶
In [21]:
correlation_values=df2.corr()['mpg']
correlation_values.abs().sort_values(ascending=False)
Out[21]:
Assuming 'mpg' as the target variable we calculate the correlation of attributes with our target variable. It seems that 'acc', only acceleration has less correlation of < 0.5 with 'mpg'¶
In [22]:
plt.figure(figsize = (15,7))
plt.title('Correlation of Attributes', y=1.05, size=19)
sns.heatmap(df2.corr(), cmap='plasma',annot=True, fmt='.2f')
Out[22]:
From the above plot the following can be inferred¶
- The correlation of 'acc' with all other attributes are less than 0.5
- The correlation of 'cyl'(cylinder) with 'disp' (displacement) and 'wt' (weight) is high
- The variable 'disp' is highly correlated to 'wt' (weight)
- The variable 'hp' (horsepower) is highly correlated with 'cyl','wt' and 'disp' attributes #### Normally if we have many attributes or variables we would drop variables that have low correlation with other attributes, for example here, the acceleration variable has low correlation with other variables but since out data set is small, it is fine not to drop a column
Scaling and Normalizaing the values¶
In [23]:
df_scaled=df2.apply(zscore)
Data Analysis and visualization¶
Checking the type of attributes¶
In [24]:
df_scaled.nunique()
Out[24]:
The attributes year, origin and cylinder are categorical while the rest are continuous¶
Five point summary¶
In [25]:
df2.describe().T
Out[25]:
In [26]:
five_point=df2.describe().T
five_point[['min','25%','50%','75%','max']]
Out[26]:
There are no negative values or any other suspiciously looking values.¶
In [27]:
sns.pairplot(df2, diag_kind='kde')
Out[27]:
In [28]:
sns.countplot(df2['origin'])
Out[28]:
In [29]:
sns.countplot(df2['cyl'])
Out[29]:
Visualizing the distribution of target variable 'mpg' using various plots¶
In [30]:
sns.distplot(df2['mpg'])
Out[30]:
In [31]:
sns.boxplot(df2['mpg'])
Out[31]:
In [32]:
sns.scatterplot(df2['mpg'],df2['wt'], hue=df2['origin'])
Out[32]:
In [33]:
sns.scatterplot(df2['mpg'],df2['wt'], hue=df2['cyl'])
Out[33]:
In [34]:
sns.swarmplot(df2['origin'],df2['mpg'], hue= df2['cyl'])
Out[34]:
In [35]:
sns.jointplot(df2['mpg'],df2['disp'], hue=df2['origin'])
Out[35]:
In [36]:
sns.jointplot(df2['mpg'],df2['hp'], hue=df2['cyl'])
Out[36]:
Machine Learning¶
In [37]:
df_scaled.isnull().sum()
Out[37]:
K means clustering¶
In [38]:
#Finding optimal no. of clusters
clusters=range(1,10)
meanDistortions=[]
for k in clusters:
model=KMeans(n_clusters=k)
model.fit(df_scaled)
prediction=model.predict(df_scaled)
meanDistortions.append(sum(np.min(cdist(df_scaled, model.cluster_centers_, 'euclidean'), axis=1)) / df_scaled.shape[0])
plt.plot(clusters, meanDistortions, 'bx-')
plt.xlabel('k')
plt.ylabel('Average distortion')
plt.title('Selecting k with the Elbow Method')
Out[38]:
We could see a clear curve and cluster 3 seems to be the elbow point¶
In [39]:
t1=datetime.datetime.now()
final_model=KMeans(3)
final_model.fit(df_scaled)
prediction=final_model.predict(df_scaled)
t2=datetime.datetime.now()
In [40]:
print('Time taken for KMeans clustering to fit and predict:', t2-t1)
In [41]:
df3=df.copy(deep =True)
df_scaled_Kmeans=df_scaled.copy(deep=True)
In [42]:
df3['GROUP']= prediction
df_scaled_Kmeans["GROUP"] = prediction
print("Groups Assigned : \n")
df3.head()
Out[42]:
In [43]:
df3Clust = df3.groupby(['GROUP'])
df3Clust.mean()
Out[43]:
The mean values of the attributes of each group is given here¶
Hierarchy Clustering¶
In [44]:
df4=df.copy(deep =True)
df_scaled_HClustering=df_scaled.copy(deep=True)
In [45]:
t3=datetime.datetime.now()
model = AgglomerativeClustering(n_clusters=3, affinity='euclidean', linkage='average')
model.fit(df_scaled_HClustering)
labels = model.labels_
t4=datetime.datetime.now()
In [46]:
print('The time taken for Hierarchy Clustering to fit and predict:', t4-t3)
In [47]:
df4['labels'] = labels
df4.head(10)
Out[47]:
In [48]:
df4 = df4.groupby(['labels'])
df4.mean()
Out[48]:
Calculating Cophenet distances¶
In [49]:
Z_average = linkage(df_scaled_HClustering, metric='euclidean', method='average')
c, coph_dists = cophenet(Z_average , pdist(df_scaled_HClustering))
print(c)
In [50]:
Z_complete = linkage(df_scaled_HClustering, metric='euclidean', method='complete')
c, coph_dists = cophenet(Z_complete , pdist(df_scaled_HClustering))
print(c)
In [51]:
Z_ward = linkage(df_scaled_HClustering, metric='euclidean', method='ward')
c, coph_dists = cophenet(Z_ward , pdist(df_scaled_HClustering))
print(c)
Form the above results, the linkage method using average distance yields highers cophenet distances around 0.75 which is closer to one.¶
On cmparing the results between Kmeans and Heirarchical clustering, the optimal number of clsuter is three and both the models yield a slightly similar results while looking at the mean of each groups produced by the models. Additionally it is surprising to note that Heirarchical took less time than Kmeans.¶
Linear Regression on the clusters¶
We are using the results generated by Heirarchical clustering¶
The optimal number of clusters required for the above data is 3, it's probably because 3 clusters are enough to explain the distortion or variability among the data points¶
Unscaled data¶
In [52]:
df5=df2.copy(deep= True)
df5.head()
Out[52]:
In [53]:
df5['labels'] = labels
In [54]:
df5.dtypes
Out[54]:
In [55]:
dfg1=df5[df5['labels']==0]
dfg2=df5[df5['labels']==1]
dfg3=df5[df5['labels']==2]
In [56]:
dfg1.head()
Out[56]:
In [57]:
dfg2.head()
Out[57]:
In [58]:
dfg3.head()
Out[58]:
In [59]:
xg1=dfg1.drop('mpg',axis=1)
yg1=dfg1[['mpg']]
xg2=dfg2.drop('mpg',axis=1)
yg2=dfg2[['mpg']]
xg3=dfg3.drop('mpg',axis=1)
yg3=dfg3[['mpg']]
In [60]:
xtrain1, xtest1, ytrain1, ytest1 = train_test_split(xg1,yg1, test_size=0.30, random_state=1)
xtrain2, xtest2, ytrain2, ytest2 = train_test_split(xg2,yg2, test_size=0.30, random_state=1)
xtrain3, xtest3, ytrain3, ytest3 = train_test_split(xg3,yg3, test_size=0.30, random_state=1)
In [61]:
regression_model = LinearRegression()
In [62]:
regression_model.fit(xtrain1, ytrain1)
print('In Sample score for group 1:', regression_model.score(xtrain1, ytrain1))
for idx, col_name in enumerate(xtrain1.columns):
print("The coefficient for {} is {}".format(col_name, regression_model.coef_[0][idx]))
In [63]:
regression_model.fit(xtrain2, ytrain2)
print('In sample score for group 2 :', regression_model.score(xtrain2, ytrain2))
for idx, col_name in enumerate(xtrain2.columns):
print("The coefficient for {} is {}".format(col_name, regression_model.coef_[0][idx]))
In [64]:
regression_model.fit(xtrain3, ytrain3)
print('In sample score for group 3 :' ,regression_model.score(xtrain3, ytrain3))
for idx, col_name in enumerate(xtrain3.columns):
print("The coefficient for {} is {}".format(col_name, regression_model.coef_[0][idx]))
Scaled Data¶
In [65]:
df_scaled_HClustering['labels']=labels
In [66]:
df_scaled_HClustering.head()
Out[66]:
In [67]:
dfg1=df_scaled_HClustering[df_scaled_HClustering['labels']==0]
dfg2=df_scaled_HClustering[df_scaled_HClustering['labels']==1]
dfg3=df_scaled_HClustering[df_scaled_HClustering['labels']==2]
In [68]:
dfg1.head()
Out[68]:
In [69]:
dfg2.head()
Out[69]:
In [70]:
dfg3.head()
Out[70]:
In [71]:
xg1=dfg1.drop('mpg',axis=1)
yg1=dfg1[['mpg']]
xg2=dfg2.drop('mpg',axis=1)
yg2=dfg2[['mpg']]
xg3=dfg3.drop('mpg',axis=1)
yg3=dfg3[['mpg']]
In [72]:
xtrain1, xtest1, ytrain1, ytest1 = train_test_split(xg1,yg1, test_size=0.30, random_state=1)
xtrain2, xtest2, ytrain2, ytest2 = train_test_split(xg2,yg2, test_size=0.30, random_state=1)
xtrain3, xtest3, ytrain3, ytest3 = train_test_split(xg3,yg3, test_size=0.30, random_state=1)
In [73]:
regression_model = LinearRegression()
In [74]:
regression_model.fit(xtrain1, ytrain1)
print('In Sample score for group 1:', regression_model.score(xtrain1, ytrain1))
for idx, col_name in enumerate(xtrain1.columns):
print("The coefficient for {} is {}".format(col_name, regression_model.coef_[0][idx]))
In [75]:
regression_model.fit(xtrain2, ytrain2)
print('In sample score for group 2 :', regression_model.score(xtrain2, ytrain2))
for idx, col_name in enumerate(xtrain2.columns):
print("The coefficient for {} is {}".format(col_name, regression_model.coef_[0][idx]))
In [76]:
regression_model.fit(xtrain3, ytrain3)
print('In sample score for group 3 :' ,regression_model.score(xtrain3, ytrain3))
for idx, col_name in enumerate(xtrain3.columns):
print("The coefficient for {} is {}".format(col_name, regression_model.coef_[0][idx]))
Inference¶
While applying linear regression on the whole data, there seems to be an increase in the In sample score due to number of data points available, this is not the same while applying linear reagression amopng clusters. But one advantage is that since the whole data is split into three clusters we can concentrate on the cluster we want rather than manually looking for a specified group of cars in the whole dataset¶
Improvement¶
More number of attributes can be added to the dataset, even if the dimension increases we can use PCA. The number of attributes and data points collected should increase distortion in the dataset which will be useful to further group the data into more clusters¶
Part II¶
Importing Data, preprocessing and EDA¶
In [77]:
df=pd.read_excel('Part2 - Company.xlsx')
In [78]:
df.head()
Out[78]:
In [79]:
print('The size of the data:', df.size)
print('No of rows in the data:', df.shape[0])
print('No of columns in the data:', df.shape[1])
In [80]:
df2=df[['A','B','C','D']]
In [81]:
df2.head()
Out[81]:
In [82]:
df2 = df2.convert_objects(convert_numeric=True)
In [83]:
df2.isnull().sum()
Out[83]:
Correlation Table¶
In [84]:
plt.figure(figsize = (15,7))
plt.title('Correlation of Attributes', y=1.05, size=19)
sns.heatmap(df2.corr(), cmap='plasma',annot=True, fmt='.2f')
Out[84]:
In [85]:
df2.nunique()
Out[85]:
All the variables are continuous while the target variable is categorical with only two values¶
Five point Summary¶
In [86]:
five_point=df2.describe().T
five_point[['min','25%','50%','75%','max']]
Out[86]:
In [87]:
sns.pairplot(df2, diag_kind='kde')
Out[87]:
In [88]:
sns.jointplot(df['A'],df['B'])
Out[88]:
In [89]:
sns.jointplot(df['C'],df['D'])
Out[89]:
In [90]:
sns.boxplot(df['A'])
Out[90]:
In [91]:
sns.boxplot(df['C'])
Out[91]:
Using Impute function for generating the missing values¶
In [92]:
imp = SimpleImputer(strategy="most_frequent")
In [93]:
df3= imp.fit_transform(df)
In [94]:
pd.DataFrame(df3).to_csv("Imputeddata.csv", index= None)
In [95]:
df2=pd.read_csv('Imputeddata.csv' )
In [96]:
df2.head()
Out[96]:
In [97]:
df2.columns=df.columns
In [98]:
df2.head()
Out[98]:
In [99]:
df.isnull().sum()
Out[99]:
In [100]:
df2.isnull().sum()
Out[100]:
In [101]:
sns.countplot(df['Quality'])
Out[101]:
Similarly even if the company collects more data with missing values we can use this function to sysnthetically generate the missing values¶
Part III¶
Import and preprocess the data¶
In [102]:
df=pd.read_csv('Part3 - vehicle.csv')
In [103]:
print('The size of the data:', df.size)
print('No of rows in the data:', df.shape[0])
print('No of columns in the data:', df.shape[1])
In [104]:
df.head()
Out[104]:
In [105]:
df.columns
Out[105]:
In [106]:
x=df.drop('class',axis=1)
y=df[['class']]
In [107]:
x.head()
Out[107]:
In [108]:
y.head()
Out[108]:
Checking for non-numerical values, removing them and filling the missing values with median values¶
In [109]:
x = x.convert_objects(convert_numeric=True)
In [110]:
x.isnull().sum()
Out[110]:
In [111]:
medianFiller = lambda x: x.fillna(x.median())
x = x.apply(medianFiller,axis=0)
In [112]:
x.isnull().sum()
Out[112]:
In [113]:
y.isnull().sum()
Out[113]:
In [114]:
(x==0).all()
Out[114]:
In [115]:
(y==0).all()
Out[115]:
In [116]:
df=y.join(x,how='left')
In [117]:
df.head()
Out[117]:
In [118]:
df.nunique()
Out[118]:
Almost all the attributes are continuous¶
EDA¶
Correlation Table¶
In [119]:
plt.figure(figsize = (15,7))
plt.title('Correlation of Attributes', y=1.05, size=19)
sns.heatmap(x.corr(), cmap='plasma',annot=True, fmt='.2f')
Out[119]:
From the correlation table the attributes from 'compactness' to 'scaled_radius_of_gyration.1' have strong correlation with other values¶
In [120]:
df.describe().T
Out[120]:
Five point Summary¶
In [121]:
five_point=x.describe().T
five_point[['min','25%','50%','75%','max']]
Out[121]:
Visualization¶
In [122]:
sns.pairplot(df, diag_kind='kde')
Out[122]:
In [123]:
sns.countplot(df['class'])
Out[123]:
In [124]:
sns.jointplot(df['compactness'],df['scatter_ratio'], hue= df['class'])
Out[124]:
In [125]:
sns.distplot(df['compactness'])
Out[125]:
In [126]:
sns.stripplot(df['pr.axis_rectangularity'], df['circularity'], hue=df['class'])
Out[126]:
In [127]:
sns.stripplot(df['max.length_aspect_ratio'], df['hollows_ratio'], hue=df['class'])
Out[127]:
Normalizing and Scaling the data¶
SVM without PCA¶
In [128]:
x_scaled=x.apply(zscore)
In [129]:
xtrain, xtest, ytrain, ytest = train_test_split(x_scaled,y, test_size=0.30, random_state=1)
In [130]:
def getAccuracy(testSet, predictions):
correct = 0
for x in range(len(testSet)):
if testSet[x]== predictions[x]:
correct += 1
return (correct/float(len(testSet))) * 100.0
In [131]:
clf = svm.SVC(gamma=0.025, C=3)
In [132]:
t1=datetime.datetime.now()
clf.fit(xtrain , ytrain)
y_pred = clf.predict(xtest)
t2=datetime.datetime.now()
In [133]:
print('Time Taken:', t2-t1)
In [134]:
ytest2=ytest.to_numpy()
np.resize(ytest2,(254,))
Out[134]:
In [135]:
print('Accuracy :', getAccuracy(ytest2 , y_pred))
In [136]:
#x_scaled=x.apply(zscore)
Applying PCA¶
In [137]:
covMatrix = np.cov(x_scaled,rowvar=False)
#print(covMatrix)
In [138]:
pca = PCA(n_components=18)
pca.fit(x_scaled)
Out[138]:
In [139]:
plt.bar(list(range(1,19)),pca.explained_variance_ratio_,alpha=0.5, align='center')
plt.ylabel('Variation explained')
plt.xlabel('eigen Value')
plt.show()
In [140]:
plt.step(list(range(1,19)),np.cumsum(pca.explained_variance_ratio_), where='mid')
plt.ylabel('Cum of variation explained')
plt.xlabel('eigen Value')
plt.show()
From the graph we can understand that eigen value of 10 explains almost 100% of the variation¶
In [141]:
pca3 = PCA(n_components=10)
pca3.fit(x_scaled)
print(pca3.components_)
print(pca3.explained_variance_ratio_)
xpca = pca3.transform(x_scaled)
In [142]:
sns.pairplot(pd.DataFrame(xpca))
Out[142]:
After applying the PCA we can understand that the variables are not related to one another and it is evident from the scatter plots and all those plots are normally distributed. It can also be noted that attribute 2 has some outliers¶
SVM with PCA¶
In [143]:
xtrain, xtest, ytrain, ytest = train_test_split(xpca,y,test_size=0.30, random_state=1)
In [144]:
t3=datetime.datetime.now()
clf.fit(xtrain , ytrain)
y_pred = clf.predict(xtest)
t4=datetime.datetime.now()
In [145]:
print('Time Taken :',t4-t3)
In [146]:
print('Accuracy:',getAccuracy(ytest2 , y_pred))
Conclusion¶
On comparing the results we can clearly see that on using PCA we were able reduce the dimensions from 18 to 10 and accuracy isn't much affected but SVM when trained with PCA performed much faster.¶
Part IV¶
Import Data and EDA¶
In [147]:
df=pd.read_csv('Part4 - batting_bowling_ipl_bat.csv')
In [148]:
df.head()
Out[148]:
In [149]:
df=df.dropna()
In [150]:
df.reset_index(drop=True, inplace= True)
In [151]:
df.head()
Out[151]:
In [152]:
print('The size of the data:', df.size)
print('No of rows in the data:', df.shape[0])
print('No of columns in the data:', df.shape[1])
In [153]:
df2=df.copy(deep=True)
In [154]:
df2.drop('Name',axis=1, inplace= True)
In [155]:
df2 = df2.convert_objects(convert_numeric=True)
In [156]:
df2.isnull().sum()
Out[156]:
In [157]:
plt.figure(figsize = (15,7))
plt.title('Correlation of Attributes', y=1.05, size=19)
sns.heatmap(df2.corr(), cmap='plasma',annot=True, fmt='.2f')
Out[157]:
In [158]:
five_point=df2.describe().T
five_point[['min','25%','50%','75%','max']]
Out[158]:
In [159]:
sns.pairplot(df2, diag_kind='kde')
Out[159]:
From the distribution plot of the attributes, the attributes are either left or right skewed¶
In [160]:
sns.jointplot(df2['SR'],df2['Runs'])
Out[160]:
We can see a postive correlation between SR and Runs at the later half where SR> 100¶
In [161]:
sns.stripplot(df['HF'],df['Runs'], jitter= True)
Out[161]:
In [162]:
sns.stripplot(df['HF'],df['Sixes'], jitter= True)
Out[162]:
In [163]:
sns.stripplot(df['HF'],df['Fours'], jitter= True)
Out[163]:
In [164]:
sns.scatterplot(df['Fours'],df['Sixes'], hue=df['HF'])
Out[164]:
In [165]:
sns.scatterplot(df['Runs'],df['SR'], hue=df['HF'])
Out[165]:
In [166]:
sns.scatterplot(df['Runs'],df['Ave'], hue=df['HF'])
Out[166]:
In [167]:
sns.boxplot(df['Ave'])
Out[167]:
In [168]:
sns.boxplot(df['Runs'])
Out[168]:
In [169]:
sns.boxplot(df['Sixes'])
Out[169]:
In [170]:
sns.boxplot(df['Fours'])
Out[170]:
From the above boxplots, neglible number of outliers are present¶
One way to rank players is by grouping players with high scores and clustering does a good job of doing that¶
Clustering and Ranking¶
In [171]:
df_scaled=df2.apply(zscore)
In [172]:
df_scaled.nunique()
Out[172]:
In [173]:
#Finding optimal no. of clusters
clusters=range(1,10)
meanDistortions=[]
for k in clusters:
model=KMeans(n_clusters=k)
model.fit(df_scaled)
prediction=model.predict(df_scaled)
meanDistortions.append(sum(np.min(cdist(df_scaled, model.cluster_centers_, 'euclidean'), axis=1)) / df_scaled.shape[0])
plt.plot(clusters, meanDistortions, 'bx-')
plt.xlabel('k')
plt.ylabel('Average distortion')
plt.title('Selecting k with the Elbow Method')
Out[173]:
Elbow Point : 5¶
In [174]:
model = AgglomerativeClustering(n_clusters=5, affinity='euclidean', linkage='average')
In [175]:
model.fit(df_scaled)
Out[175]:
In [176]:
df['labels'] = model.labels_
df.head(10)
Out[176]:
In [177]:
dfcluster = df.groupby(['labels'])
The ninety players in the datasets are now split and ranked into five categories. Based on the mean scores, it'll be easier to know players belonging to which cluster have excellent performance.¶
In [178]:
df3= dfcluster.mean()
df3
Out[178]:
Creating a separate dataframe for each cluster¶
- The players with label 3 are ranked first,
- Label 4 are ranked second
- Label 2 are ranked third
In [179]:
Rank1=df5[df5['labels']==3]
Rank2=df5[df5['labels']==4]
Rank3=df5[df5['labels']==2]
Rank4=df[df['labels']==0]
Rank5=df[df['labels']==1]
In [180]:
Z_average = linkage(df_scaled, metric='euclidean', method='average')
c, coph_dists = cophenet(Z_average , pdist(df_scaled))
print('cophenet index:', c)
Part V¶
1. Dimensionality reduction techniques implemented in pyhton include¶
- PCA
- SVD (Singular Value decomposition) and
- LDA, apart from dimensionality reduction, it can also be used in classification algorithm.
2. Yes Dimensionality reduction can also be applied in multimedia data like images and videos. These techniques are widely used for image compression.¶
Example: Using PCA to compress images¶
In [181]:
image_raw = imread("test.jpg")
print(image_raw.shape)
# Displaying the image
plt.figure(figsize=[12,8])
plt.imshow(image_raw)
Out[181]:
In [182]:
image_sum = image_raw.sum(axis=2)
print(image_sum.shape)
image_bw = image_sum/image_sum.max()
print(image_bw.max())
plt.figure(figsize=[12,8])
plt.imshow(image_bw, cmap=plt.cm.gray)
Out[182]:
In [183]:
pca = PCA()
pca.fit(image_bw)
# Getting the cumulative variance
var_cumu = np.cumsum(pca.explained_variance_ratio_)*100
# How many PCs explain 95% of the variance?
k = np.argmax(var_cumu>95)
print("Number of components explaining 95% variance: "+ str(k))
plt.figure(figsize=[10,5])
plt.title('Cumulative Explained Variance explained by the components')
plt.ylabel('Cumulative Explained variance')
plt.xlabel('Principal components')
plt.axvline(x=k, color="k", linestyle="--")
plt.axhline(y=95, color="r", linestyle="--")
ax = plt.plot(var_cumu)
In [184]:
ipca = IncrementalPCA(n_components=k)
image_recon = ipca.inverse_transform(ipca.fit_transform(image_bw))
# Plotting the reconstructed image
plt.figure(figsize=[12,8])
plt.imshow(image_recon,cmap = plt.cm.gray)
Out[184]:
In [ ]:
In [ ]: